package Http.httpclient;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.*;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.util.EntityUtils;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.*;

public class HttpClient {


    /**
     * 转化Caoliu返回的html
     */
    public static String parsingHtmlCaoliu(CloseableHttpResponse responce) throws Exception {
        String html = "";
        HttpEntity entity = responce.getEntity();
        if (entity != null) {
            html = EntityUtils.toString(entity, "UTF-8");
            html = html.replace("&nbsp;", " ");
        }
        return html;
    }

    /**
     * 通用的获取Caoliu html的get方法
     */

    /**
     * 通用的获取url 302 跳转地址 的post方法
     */
    public static String postDirect302Url(String url, Map<String, String> map) throws IOException {
        // https://dx.doi.org 此post 专用
        // referer: https://dx.doi.org/
        // origin: https://dx.doi.org
        BasicCookieStore cookieStore = new BasicCookieStore();
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "https://dx.doi.org/", "dx.doi.org", false);
        config.setProxy(false);
        String html = "";
        CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
        T:
        for (int i = 1; i <= 3; i++) {
            HttpPost httppost;
            CloseableHttpResponse response;
            try {
                httppost = new HttpPost(config.getUrl());
                List<NameValuePair> nvps = new ArrayList<>();
                if (map != null) {
                    for (Map.Entry<String, String> entry : map.entrySet()) {
                        nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
                    }
                }
                httppost.setEntity(new UrlEncodedFormEntity(nvps, "utf8"));
                httppost.setConfig(config.getRequestConfig());
                httppost.setHeader("Content-type", "application/x-www-form-urlencoded");
                httppost.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httppost.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httppost.addHeader(HttpHeaders.HOST, config.getHost());
                }
                response = httpClient.execute(httppost);
                int resStatu = response.getStatusLine().getStatusCode();
//                SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "状态码" + resStatu);
                if (resStatu == HttpStatus.SC_MOVED_PERMANENTLY || resStatu == HttpStatus.SC_MOVED_TEMPORARILY) {
                    Header[] allHeaders = response.getAllHeaders();
                    for (Header allHeader : allHeaders) {
                        String name = allHeader.getName();
                        if (name.toLowerCase().contains("location")) {
                            String value = allHeader.getValue();
                            if (StringUtils.isNotEmpty(value)) {
                                html = value;
                                break T;
                            }

                        }
                    }
                } else if (resStatu == HttpStatus.SC_NOT_FOUND) {
                    html = "404";
                    break T;
                } else if (resStatu == HttpStatus.SC_BAD_REQUEST) {
                    // 10.2549/jospt.2017.5544 部分doi错误 导致
                    html = "404";
                    break T;
                }
            } catch (Exception e) {
//                SmallTool.printMessError("查看异常的字符串" + e.toString());
                if (e.toString().contains("java.net.UnknownHostException")) {

                }
                if (e.toString().contains("connect time out")) {
                }
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


    /**
     * 通用的获取html的get方法 无代理
     */
    public static String getHtmlNoProxy(BasicCookieStore cookieStore, HttpClientConfig config) throws IOException {
//        //SmallTool.printMess("当前的地址:" + config.getUrl());
        String html = "";
        for (int i = 1; i <= 4; i++) {
            CloseableHttpClient httpClient = HttpClientBuilder.create()
                    .setDefaultCookieStore(cookieStore)
                    .setDefaultCredentialsProvider(config.getCredsProvider())
                    .build();
            HttpGet httpget = null;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                //添加cookie 测试
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        break;
                    }
                } else {
                    SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                    SmallTool.SleepMillis(100000L);
                }
            } catch (Exception e) {
                String ers = e.getMessage();
                if (ers != null) {
                    SmallTool.printMessError("异常信息++" + ers);
                    if (ers.contains("Connection refused: connect") || ers.contains("timed out")) {
                        SmallTool.printMessError("代理异常 用直连测试");
                        config.setProxy(false);
                    }
                } else {
                    SmallTool.printMessError("url++" + config.getUrl());
                }
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


    private static boolean savePicToDisk(InputStream in, String dirPath, String filePath) {
        boolean result = false;
        try {
            File dir = new File(dirPath);
            if (dir == null || !dir.exists()) {
                dir.mkdirs();
            }
            //文件真实路径
            String realPath = dirPath.concat(filePath);
            File file = new File(realPath);
            if (file == null || !file.exists()) {
                file.createNewFile();
            }

            FileOutputStream fos = new FileOutputStream(file);
            byte[] buf = new byte[10240];
            int len = 0;
            while ((len = in.read(buf)) != -1) {
                fos.write(buf, 0, len);
            }
            fos.flush();
            fos.close();
            result = true;

        } catch (IOException e) {
            SmallTool.printMessError("异常信息++" + e.getMessage());
        } finally {
            try {
                in.close();
            } catch (IOException e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            }
        }
        return result;
    }

    /**
     * 发送 http post 请求，参数以原生字符串进行提交
     *
     * @param config
     * @param encode
     * @return
     */
    public static HttpResponse httpPostRaw(HttpClientConfig config, String stringJson, Map<String, String> headers, String encode) {
        HttpResponse response = null;
        if (encode == null) {
            encode = "utf-8";
        }
        //HttpClients.createDefault()等价于 HttpClientBuilder.create().build();
        CloseableHttpClient closeableHttpClient = HttpClients.createDefault();
        HttpPost httpost = new HttpPost(config.getUrl());

        //设置header
        httpost.setHeader("Content-type", "application/json");
        if (headers != null && headers.size() > 0) {
            for (Map.Entry<String, String> entry : headers.entrySet()) {
                httpost.setHeader(entry.getKey(), entry.getValue());
            }
        }
        //组织请求参数
        StringEntity stringEntity = new StringEntity(stringJson, encode);
        httpost.setEntity(stringEntity);
        String content = null;
        CloseableHttpResponse httpResponse = null;
        try {
            //响应信息
            httpResponse = closeableHttpClient.execute(httpost);
            HttpEntity entity = httpResponse.getEntity();
            Header contentType = entity.getContentType();
            if (contentType != null) {
                HeaderElement[] elements = contentType.getElements();
                for (HeaderElement element : elements) {
                    if (element.getName().equals("application/nbib")) {
                        // 执行下载
                        InputStream in = entity.getContent();
                        boolean b = savePicToDisk(in, config.getDirPath(), config.getFilePath());
                        SmallTool.printMess("图片保存成功");
                        SmallTool.printMess("路径+++" + config.getDirPath() + config.getFilePath());
                        if (in != null) {
                            break;
                        }
                    }
                }
            }

            content = EntityUtils.toString(entity, encode);
            System.out.println(content);
            //            response.setBody(content);
//            response.setHeaders(httpResponse.getAllHeaders());
//            response.setReasonPhrase(httpResponse.getStatusLine().getReasonPhrase());
//            response.setStatusCode(httpResponse.getStatusLine().getStatusCode());
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                httpResponse.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {  //关闭连接、释放资源
            closeableHttpClient.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return response;
    }


    // 通过json 发送数据
    public static String postHtmlByJsonStr(BasicCookieStore cookieStore, String json, HttpClientConfig config) throws IOException {
//        SmallTool.printMess("httpclient里面的地址" + config.getUrl());
        String html = "";
        for (int i = 1; i <= 4; i++) {
            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
            HttpPost httppost = null;
            long time1 = System.currentTimeMillis();
            CloseableHttpResponse response;
            try {
                httppost = new HttpPost(config.getUrl());

                StringEntity stringEntity = new StringEntity(json, "utf-8");
                httppost.setEntity(stringEntity);
//                httppost.setEntity(new StringEntity(json, "utf8"));
                httppost.setConfig(config.getRequestConfig());
                httppost.setHeader("Content-type", "application/json; charset=UTF-8");
                httppost.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httppost.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httppost.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httppost.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httppost);
                int resStatu = response.getStatusLine().getStatusCode();
                SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "状态码" + resStatu);
                //获得相应实体
                if (resStatu == HttpStatus.SC_OK) {
                    //判断如果是下载则走下载程序
                    if (1 == 4) {
                        HttpEntity entity = response.getEntity();
                        InputStream in = entity.getContent();
                        boolean b = savePicToDisk(in, config.getDirPath(), config.getFilePath());
                        SmallTool.printMess("图片保存成功");
                        SmallTool.printMess("路径+++" + config.getDirPath() + config.getFilePath());
                        if (in != null) {
                            break;
                        }
                    } else {
                        html = HttpClientUtil.parsingHtml(response);
                        if (html != null) {
                            break;
                        }

                    }


                } else {
                    SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                    SmallTool.SleepMillis(100000L);
                }
            } catch (Exception e) {
                SmallTool.printMessError("查看异常的字符串" + e.toString());
                if (e.getMessage().contains("Connection refused: connect")) {
                    SmallTool.printMessError("代理异常 用直连测试");
                    config.setProxy(false);
                }

            } finally {
                httpClient.close();
            }
        }
        return html;
    }

    /**
     * 通用的获取html的post方法
     */
    public static String postHtml(BasicCookieStore cookieStore, Map<String, String> map, HttpClientConfig config) throws IOException {
//        SmallTool.printMess("httpclient里面的地址" + config.getUrl());
        String html = "";
        for (int i = 1; i <= 4; i++) {
            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
            HttpPost httppost = null;
            long time1 = System.currentTimeMillis();
            CloseableHttpResponse responce;
            try {
                httppost = new HttpPost(config.getUrl());
                List<NameValuePair> nvps = new ArrayList<>();
                if (map != null) {
                    for (Map.Entry<String, String> entry : map.entrySet()) {
                        nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
                    }
                }
                if (StringUtils.isNotEmpty(config.getPostDate())) {
                    httppost.setEntity(new StringEntity(config.getPostDate(), "utf-8"));
                }
                if (nvps.size() > 0) {
                    httppost.setEntity(new UrlEncodedFormEntity(nvps, "utf8"));
                }
                httppost.setConfig(config.getRequestConfig());
                httppost.setHeader("Content-type", "application/x-www-form-urlencoded");
                httppost.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httppost.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httppost.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    for (Map.Entry<String, String> entry : config.getHeaderMap().entrySet()) {
                        httppost.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                responce = httpClient.execute(httppost);
                int resStatu = responce.getStatusLine().getStatusCode();
                SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "状态码" + resStatu);
                //获得相应实体
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(responce);
                    if (html != null) {
                        break;
                    }
                } else {
                    SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                    SmallTool.SleepMillis(100000L);
                }
            } catch (Exception e) {
                SmallTool.printMessError("查看异常的字符串" + e.toString());
                e.printStackTrace();
                if (e.getMessage().contains("Connection refused: connect")) {
                    SmallTool.printMessError("代理异常 用直连测试");
                    config.setProxy(false);
                }
//                if (e.toString().contains("java.net.UnknownHostException")) {
//
//                }
//                if (e.toString().contains("connect time out")) {
//                }
            } finally {
                httpClient.close();
            }
        }
        return html;
    }

    /**
     * 通用的下载图片get方法
     */
    public static void getDownImage(HttpClientConfig config) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        File file = new File(config.getDirPath() + config.getFilePath());
        if (file.exists()) {
//            SmallTool.printMess("存在不下载 嘻嘻" + config.getFilePath());
        } else {
//            SmallTool.printMess("当前的地址:" + config.getUrl());
            for (int i = 1; i <= 4; i++) {
                CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
                HttpGet httpget = null;
                long time1 = System.currentTimeMillis();
                CloseableHttpResponse response;
                try {
                    httpget = new HttpGet(config.getUrl());
                    httpget.setConfig(config.getRequestConfig());
                    httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                    if (config.getHeaderMap().size() > 0) {
                        Map<String, String> headerMap = config.getHeaderMap();
                        Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                        for (Map.Entry<String, String> entry : entries) {
                            httpget.addHeader(entry.getKey(), entry.getValue());
                        }
                    }
                    response = httpClient.execute(httpget);
                    int resStatu = response.getStatusLine().getStatusCode();
                    if (resStatu == 302) {
                        LogWriteUtil.WriteLog(config.getFilePath() + "没有地址是" + config.getUrl(), "");
                        SmallTool.printMess(resStatu);
                        break;
                    }
                    //SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "--状态码" + resStatu);
                    if (resStatu == HttpStatus.SC_OK) {
                        HttpEntity entity = response.getEntity();

                        InputStream in = entity.getContent();
                        boolean b = savePicToDisk(in, config.getDirPath(), config.getFilePath());
                        SmallTool.printMess("图片保存成功");
                        SmallTool.printMess("路径+++" + config.getDirPath() + config.getFilePath());
                        if (in != null) {
                            break;
                        }
                    } else {
                        SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                        SmallTool.SleepMillis(100000L);
                    }
                } finally {
                    httpClient.close();
                }
            }
        }
    }



    /**
     * 通用的下载图片get方法 返回http 状态码
     */
    public static Integer getDownImageCode(HttpClientConfig config) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        Integer code = 404;
        File file = new File(config.getDirPath() + config.getFilePath());
        if (file.exists()) {
            code = 200;
            SmallTool.printMess("存在不下载 嘻嘻" + config.getFilePath());
        } else {
            SmallTool.printMess("当前的地址:" + config.getUrl());
            for (int i = 1; i <= 2; i++) {
                CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
                HttpGet httpget = null;
                long time1 = System.currentTimeMillis();
                CloseableHttpResponse response;
                try {
                    httpget = new HttpGet(config.getUrl());
                    httpget.setConfig(config.getRequestConfig());
                    httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                    if (config.getHeaderMap().size() > 0) {
                        Map<String, String> headerMap = config.getHeaderMap();
                        Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                        for (Map.Entry<String, String> entry : entries) {
                            httpget.addHeader(entry.getKey(), entry.getValue());
                        }
                    }
                    response = httpClient.execute(httpget);
                    int resStatu = response.getStatusLine().getStatusCode();
                    if (resStatu == 302) {
                        LogWriteUtil.WriteLog(config.getFilePath() + "没有地址是" + config.getUrl(), "");
                        SmallTool.printMess(resStatu);
                        break;
                    }
                    SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "--状态码" + resStatu);
                    if (resStatu == HttpStatus.SC_OK) {
                        HttpEntity entity = response.getEntity();
                        InputStream in = entity.getContent();
                        boolean b = savePicToDisk(in, config.getDirPath(), config.getFilePath());
                        if (b) {
                            SmallTool.printMess("图片保存成功");
                            SmallTool.printMess("路径+++" + config.getDirPath() + config.getFilePath());
                            code = resStatu;
                        }

                        if (in != null) {
                            break;
                        }
                    } else {
                        SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                        code = resStatu;
                        return code;
                    }
                } catch (Exception e) {
                    SmallTool.printMessError("下载异常++" + config.getUrl());
//                    e.printStackTrace();
//                    if (e!=null&&e.getMessage().contains("Connection refused: connect")) {
////                        SmallTool.printMessError("代理异常 用直连测试");
////                        config.setProxy(false);
//                    }
                } finally {
                    httpClient.close();
                }
            }
        }
        return code;
    }



    /**
     * 通用的下载图片get方法
     */
    public static void getImage(BasicCookieStore cookieStore, HttpClientConfig config, String dirPath, String filePath) throws IOException {
        File file = new File(dirPath + filePath);
        if (file.exists()) {
//            SmallTool.printMess("存在不下载 嘻嘻" + filePath);
        } else {
            //SmallTool.printMess("当前的地址:" + config.getUrl());
            for (int i = 1; i <= 6; i++) {
                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());
                CloseableHttpClient httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider()).build();
                HttpGet httpget = null;
                long time1 = System.currentTimeMillis();
                CloseableHttpResponse response;
                try {
                    httpget = new HttpGet(config.getUrl());
                    httpget.setConfig(config.getRequestConfig());
                    httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                    if (config.getHeaderMap().size() > 0) {
                        Map<String, String> headerMap = config.getHeaderMap();
                        Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                        for (Map.Entry<String, String> entry : entries) {
                            httpget.addHeader(entry.getKey(), entry.getValue());
                        }
                    }
                    response = httpClient.execute(httpget);
                    int resStatu = response.getStatusLine().getStatusCode();
                    if (resStatu == 302) {
                        LogWriteUtil.WriteLog(filePath + "没有地址是" + config.getUrl(), "");
                        SmallTool.printMess(resStatu);
                        break;
                    }
                    //SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "--状态码" + resStatu);
                    if (resStatu == HttpStatus.SC_OK) {
                        HttpEntity entity = response.getEntity();
                        InputStream in = entity.getContent();
                        boolean b = savePicToDisk(in, dirPath, filePath);
                        //SmallTool.printMess("图片保存成功");
//                        SmallTool.printMess("路径+++" + dirPath + filePath);
                        if (in != null) {
                            break;
                        }
                    } else {
                        SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                        SmallTool.SleepMillis(100000L);
                    }
                } catch (Exception e) {
                    SmallTool.printMessError("下载异常++" + config.getUrl());
                    if (e.getMessage().contains("Connection refused: connect")) {
                        SmallTool.printMessError("代理异常 用直连测试");
                        config.setProxy(false);
                    }

                } finally {
                    httpClient.close();
                }
            }
        }
    }


    public static CloseableHttpClient createSSLClientDefault(BasicCookieStore basicCookieStore, HttpClientConfig clientConfig) {
        HttpHost httpHost = null;
        CloseableHttpClient httpClient;
        dealHttpClientConfig(clientConfig, basicCookieStore);
        basicCookieStore = clientConfig.getCookieStore();
        clientConfig = clientConfig.build();
        if (clientConfig.isProxy()) {
            httpHost = new HttpHost(clientConfig.getProxyIp(), clientConfig.getProxyPort());
        } else {

        }
        if (clientConfig.getProxyInfo() != null) {
            httpHost = new HttpHost(clientConfig.getProxyInfo().getProxyIp(), Integer.parseInt(clientConfig.getProxyInfo().getProxyPort()));
        }
        //设置代理
        if (clientConfig.getSslUse()) {
            try {
                //使用 loadTrustMaterial() 方法实现一个信任策略，信任所有证书
                // 信任所有
                SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true).build();
                //NoopHostnameVerifier类:  作为主机名验证工具，实质上关闭了主机名验证，它接受任何
                //有效的SSL会话并匹配到目标主机。
                HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE;
                SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
                if (clientConfig.isProxy()) {
                    SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(clientConfig.getTIME_OUT()).build();
                    CloseableHttpClient build = HttpClients.custom()
                            .setProxy(httpHost)
                            .setDefaultCookieStore(basicCookieStore)
                            .setDefaultSocketConfig(socketConfig)
                            .setDefaultCredentialsProvider(clientConfig.getCredsProvider())
                            .setSSLSocketFactory(sslsf).build();
                    return build;
                } else {
                    CloseableHttpClient build = null;
                    build = HttpClients.custom().setDefaultCookieStore(basicCookieStore).setDefaultCredentialsProvider(clientConfig.getCredsProvider()).setSSLSocketFactory(sslsf).build();
                    return build;
                }
            } catch (KeyManagementException e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } catch (NoSuchAlgorithmException e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } catch (KeyStoreException e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            }
        } else {
            if (clientConfig.isProxy()) {
                SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(clientConfig.TIME_OUT).build();
                httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultSocketConfig(socketConfig)
                        .setDefaultCookieStore(clientConfig.getCookieStore())
                        .setDefaultCredentialsProvider(clientConfig.getCredsProvider())
                        .build();
                return httpClient;
            } else {
                if (UrlEntity.DailiFlag) {
                    SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(clientConfig.TIME_OUT).build();
                    httpClient = HttpClientBuilder.create()
                            .setProxy(httpHost)
                            .setDefaultSocketConfig(socketConfig)
                            .setDefaultCookieStore(clientConfig.getCookieStore())
                            .setDefaultCredentialsProvider(clientConfig.getCredsProvider())
                            .build();
                    return httpClient;
                } else {
                    httpClient = HttpClientBuilder.create()
                            .setDefaultCookieStore(clientConfig.getCookieStore())
                            .setDefaultCredentialsProvider(clientConfig.getCredsProvider())
                            .build();

                    return httpClient;
                }
            }


        }
        return HttpClients.createDefault();
    }

    //如果config 里面有cookie 转换成basicCookieStore
    public static void dealHttpClientConfig(HttpClientConfig config, BasicCookieStore basicCookieStore) {
        if (isNotNull(config.getCookie())) {
            SmallTool.printMess("cookie不为空 给basicCookieStore赋值");
            BasicCookieStore cookieStore = new BasicCookieStore();
            String[] split = config.getCookie().split(";");
            for (String s : split) {
                String[] keyValue = s.split("=");
                if (keyValue.length == 2) {
                    BasicClientCookie cookie = new BasicClientCookie(keyValue[0], keyValue[1]);
                    cookie.setDomain(config.getHost());
                    cookie.setPath("/");
                    cookieStore.addCookie(cookie);
                }
            }
            basicCookieStore = cookieStore;
            config.setCookieStore(cookieStore);
        }
    }

    public static boolean isNotNull(Object file) {
        if (file != null && !file.toString().trim().equals("") && !file.toString().trim().equals("null") && !file.toString().trim().equals("[]]")) {
            return true;
        } else {
            return false;
        }
    }



    //如果config 里面有cookie 转换成basicCookieStore
    public static BasicCookieStore dealCookieToBasicCookieStore(String cookieStr, String host, BasicCookieStore basicCookieStore) {
        BasicCookieStore res = new BasicCookieStore();
        if (isNotNull(cookieStr)) {
            SmallTool.printMess("cookie不为空 给basicCookieStore赋值");
            String[] split = cookieStr.split(";");
            for (String s : split) {
                String[] keyValue = s.split("=");
                if (keyValue.length == 2) {
                    BasicClientCookie cookie = new BasicClientCookie(keyValue[0], keyValue[1]);
                    cookie.setDomain(host);
                    cookie.setPath("/");
                    basicCookieStore.addCookie(cookie);
                }
            }
            return basicCookieStore;
        }
        return res;
    }


    /**
     * 通用的获取html的get方法
     *
     * @param url
     * @param param 用于草榴下载编码用
     * @return
     * @throws IOException
     */
    public static String getHtmlDirect(String url, String param) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "", "", true);
        String html = "";
        config.setProxy(UrlEntity.DailiFlag);
        CloseableHttpClient httpClient;
        for (int i = 1; i <= 4; i++) {
            if (config.isProxy()) {
                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());
                httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();

            } else {
                httpClient = HttpClientBuilder.create()
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();
            }
//            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    if (StringUtils.isNotEmpty(param) && param.equals("caoliu")) {
                        html = parsingHtmlCaoliu(response);
                    } else {
                        html = HttpClientUtil.parsingHtml(response);

                    }
                    if (html != null) {
                        break;
                    }
                } else {
                    SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                    SmallTool.SleepMillis(100000L);
                }
            } catch (Exception e) {
                String ers = e.getMessage();
                if (ers != null) {
                    SmallTool.printMessError("异常信息++" + ers);
                    if (ers.contains("Connection refused: connect") || ers.contains("timed out")) {
                        SmallTool.printMessError("代理异常 用直连测试");
                        config.setProxy(false);
                    }
                } else {
                    SmallTool.printMessError("url++" + url);
                }
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


    /**
     * 通用的获取html的get方法
     */
    public static String getHtmlDirectProxyWithCookie(String url, BasicCookieStore cookieStore) throws IOException {
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "", "", true);
        config.setProxy(true);
        config.setSslUse(true);
        String html = "";
        CloseableHttpClient httpClient;

        for (int i = 1; i <= 4; i++) {

            if (config.isProxy()) {
                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());

                httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();

            } else {
                httpClient = HttpClientBuilder.create()
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();
            }
//            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);

            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        break;
                    }
                } else {
                    SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                    SmallTool.SleepMillis(100000L);
                }
            } catch (Exception e) {
                String ers = e.getMessage();
                if (ers != null) {
                    SmallTool.printMessError("异常信息++" + ers);
                    if (ers.contains("Connection refused: connect") || ers.contains("timed out")) {
                        SmallTool.printMessError("代理异常 用直连测试");
                        config.setProxy(false);
                    }
                } else {
                    SmallTool.printMessError("url++" + url);
                }
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


    /**
     * 通用的获取html的get方法
     */
    public static String getHtmlDirectProxy(String url) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "", "", true);
        config.setProxy(true);
        config.setSslUse(true);
        String html = "";
        CloseableHttpClient httpClient;

        for (int i = 1; i <= 2; i++) {

            if (config.isProxy()) {
                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());

                httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();

            } else {
                httpClient = HttpClientBuilder.create()
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();
            }
//            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);

            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        break;
                    }
                } else {
                    SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                    SmallTool.SleepMillis(100000L);
                }
            } catch (Exception e) {
                String ers = e.getMessage();
                if (ers != null) {
                    SmallTool.printMessError("异常信息++" + ers);
                    if (ers.contains("Connection refused: connect") || ers.contains("timed out")) {
                        SmallTool.printMessError("代理异常 用直连测试");
                        config.setProxy(false);
                    }
                } else {
                    SmallTool.printMessError("url++" + url);
                }
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


    /**
     * 通用的获取html的get方法
     */
    public static String getHtmlDirectProxy(String url, BasicCookieStore cookieStore) throws IOException {
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "", "", true);
        config.setProxy(true);
        config.setSslUse(true);
        String html = "";
        CloseableHttpClient httpClient;

        for (int i = 1; i <= 4; i++) {

            if (config.isProxy()) {
                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());

                httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();

            } else {
                httpClient = HttpClientBuilder.create()
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();
            }
//            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);

            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        break;
                    }
                } else {
                    SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                    SmallTool.SleepMillis(100000L);
                }
            } catch (Exception e) {
                String ers = e.getMessage();
                if (ers != null) {
                    SmallTool.printMessError("异常信息++" + ers);
                    if (ers.contains("Connection refused: connect") || ers.contains("timed out")) {
                        SmallTool.printMessError("代理异常 用直连测试");
                        config.setProxy(false);
                    }
                } else {
                    SmallTool.printMessError("url++" + url);
                }

            } finally {
                httpClient.close();
            }
        }
        if (StringUtils.isNotEmpty(html) && html.contains("由於版權限制")) {
            SmallTool.printMessError("由於版權限制+++" + url);
            return "";
        }
        return html;
    }

    /**
     * 通用的获取html的get方法
     */
    public static String getHtmlDirect(String url) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "", "", true);
        config.setProxy(false);
        String html = "";
        CloseableHttpClient httpClient;

        for (int i = 1; i <= 4; i++) {

//            if (config.isProxy()) {
//                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());
//
//                httpClient = HttpClientBuilder.create()
//                        .setProxy(httpHost)
//                        .setDefaultCookieStore(cookieStore)
//                        .setDefaultCredentialsProvider(config.getCredsProvider())
//                        .build();
//
//            } else {
//                httpClient = HttpClientBuilder.create()
//                        .setDefaultCookieStore(cookieStore)
//                        .setDefaultCredentialsProvider(config.getCredsProvider())
//                        .build();
//            }
            httpClient = HttpClient.createSSLClientDefault(cookieStore, config);

            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
//                httpget.addHeader("sec-ch-ua", "\"Chromium\";v=\"94\", \"Microsoft Edge\";v=\"94\", \";Not A Brand\";v=\"99\"");
//                httpget.addHeader("sec-ch-ua-platform", "\"Windows\"");
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        break;
                    }
                } else {
                    SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                    SmallTool.SleepMillis(100000L);
                }
            } catch (Exception e) {
                String ers = e.getMessage();
                if (ers != null) {
                    SmallTool.printMessError("异常信息++" + ers);
                    if (ers.contains("Connection refused: connect") || ers.contains("timed out")) {
                        SmallTool.printMessError("代理异常 用直连测试");
                        config.setProxy(false);
                    }
                } else {
                    SmallTool.printMessError("url++" + url);
                }
            } finally {
                httpClient.close();
            }
        }
        return html;
    }

    /**
     * 通用的获取html的get方法
     */
    public static Map<String, Object> getHtml(BasicCookieStore cookieStore, HttpClientConfig config) throws IOException {
        String html = "";
        int code = 302;
        Map<String, Object> map = new HashMap<>();
        for (int i = 1; i <= 2; i++) {
            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);

            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (!StringUtils.isEmpty(config.getAccept())) {
                    httpget.addHeader(HttpHeaders.ACCEPT, config.getAccept());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        code = HttpStatus.SC_OK;
                        map.put("code", code);
                        map.put("html", html);
                        return map;
                    }
                } else {
                    SmallTool.printMessError("状态码不是200 状态码是++" + resStatu);
                    SmallTool.SleepMillis(30000L);
                    map.put("code", resStatu);
                    map.put("html", html);
                    return map;
                }
            } catch (Exception e) {
                String ers = e.getMessage();
                if (ers != null) {
                    SmallTool.printMessError("异常信息++" + ers);
                    if (ers.contains("Connection refused: connect") || ers.contains("timed out")) {
                        SmallTool.printMessError("代理异常 用直连测试");
                        config.setProxy(false);
                    }
                } else {
                    SmallTool.printMessError("url++" + config.getUrl());
                }
            } finally {
                httpClient.close();
            }
        }
        map.put("code", code);
        map.put("html", html);
        return map;
    }

    /**
     * 通用的获取javdb html的get 304 返回网页方法
     */
//    public static String getHtmlJAVDB(BasicCookieStore cookieStore, HttpClientConfig config) throws IOException {
//        String html = "";
//        for (int i = 1; i <= 4; i++) {
//            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
//            HttpGet httpget;
//            CloseableHttpResponse response;
//            try {
//                httpget = new HttpGet(config.getUrl());
//                httpget.setConfig(config.getRequestConfig());
//                httpget.addHeader(HttpHeaders.USER_AGENT, HttpClientConfig.USER_AGENT);
//                if (!StringUtils.isEmpty(config.getRefer())) {
//                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
//                }
//                if (!StringUtils.isEmpty(config.getHost())) {
//                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
//                }
//                if (!StringUtils.isEmpty(config.getAccept())) {
//                    httpget.addHeader(HttpHeaders.ACCEPT, config.getAccept());
//                }
//                if (config.getHeaderMap().size() > 0) {
//                    Map<String, String> headerMap = config.getHeaderMap();
//                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
//                    for (Map.Entry<String, String> entry : entries) {
//                        httpget.addHeader(entry.getKey(), entry.getValue());
//                    }
//                }
//                response = httpClient.execute(httpget);
//                int resStatu = response.getStatusLine().getStatusCode();
//                if (resStatu == HttpStatus.SC_OK || resStatu == HttpStatus.SC_NOT_MODIFIED) {
//                    html = HttpClientUtil.parsingHtml(response);
//                    if (html != null) {
//                        break;
//                    }
//                }
//            } catch (Exception e) {
//                String ers = e.getMessage();
//                if (ers != null) {
//                    SmallTool.printMessError("异常信息++" + ers);
//                    if (ers.contains("Connection refused: connect") || ers.contains("timed out")) {
//                        SmallTool.printMessError("代理异常 用直连测试");
//                        config.setProxy(false);
//                    }
//                } else {
//                    SmallTool.printMessError("url++" + config.getUrl());
//                }
//            } finally {
//                httpClient.close();
//            }
//        }
//        return html;
//    }


}
